﻿Imports System.IO
Imports System.Net
Imports System.Text
Imports System.Text.RegularExpressions
Imports System.Data.Entity
Imports System.ComponentModel.DataAnnotations

Public Class Form1
    Dim isGetAll As Boolean

    Function getRegex(txt, pattern) As MatchCollection
        Return Regex.Matches(txt, pattern, RegexOptions.Singleline Or RegexOptions.IgnoreCase Or RegexOptions.Multiline Or RegexOptions.IgnorePatternWhitespace)
    End Function
    Function getPage(url, Optional postData = "") As String
        Dim cookieContainer As New CookieContainer
        cookieContainer.Add(New Cookie("PHPSESSID", "3e1dedcba81c8d75fe94e83fcc11890e", "/", "ppd.disdik-kotasmg.org"))

        Dim request As HttpWebRequest = WebRequest.Create(url)
        request.CookieContainer = cookieContainer
        request.UserAgent = "Mozilla/5.0 (Windows NT 6.1; rv:13.0) Gecko/20100101 Firefox/13.0.1 AlexaToolbar/alxf-2.15"

        Dim dataStream As Stream

        If postData <> "" Then
            request.Method = "POST"
            request.ContentType = "application/x-www-form-urlencoded"
            Dim byteArray As Byte() = Encoding.UTF8.GetBytes(postData)
            request.ContentLength = byteArray.Length

            dataStream = request.GetRequestStream()
            dataStream.Write(byteArray, 0, byteArray.Length)
            dataStream.Close()
        End If

        Dim response As WebResponse = request.GetResponse()
        'Console.WriteLine(CType(response, HttpWebResponse).StatusDescription)

        dataStream = response.GetResponseStream()
        Dim reader As New StreamReader(dataStream)
        Dim responseFromServer As String = reader.ReadToEnd()

        reader.Close()
        dataStream.Close()
        response.Close()

        Return responseFromServer
    End Function

    Function getDataSiswa(SekolahID As String, Optional offset As Integer = 0) As List(Of Siswa)
        Dim page As String = getPage("http://ppd.disdik-kotasmg.org/v12/smp/ajax/listRekap", "cassila=3e1dedcba81c8d75fe94e83fcc11890e&sc=" & SekolahID & "&offset=" & offset & "&st=x&url_=http://ppd.disdik-kotasmg.org/v12/smp/ajax/listRekap/")
        Dim siswaDom = getRegex(page, "<tr[^>]*>.*?<td[^>]*>.*?</td>.*?<td[^>]*>.*?<font[^>]*>(?<KPUN>.*?)</font>.*?</td>.*?<td[^>]*><span>(?<NamaSiswa>.*?)</span></td>.*?<td[^>]*><span>(?<AsalSekolah>.*?)</span></td>.*?<td[^>]*>.*?(?<Nilai>\d+(\.\d{0,2})?).*?</td>.*?<td[^>]*>.*?<div[^>]*>(?<Verifikasi>.*?)</div>.*?</td>.*?<td[^>]*>.*?<span>(?<Pilihan>.*?)</span>.*?</td>.*?</tr>")
        Dim siswaList As New List(Of Siswa)

        For Each item As Match In siswaDom
            Dim siswa As New Siswa
            siswa.KPUN = item.Groups("KPUN").Value
            siswa.NamaSiswa = item.Groups("NamaSiswa").Value
            siswa.SekolahAsal = item.Groups("AsalSekolah").Value
            Dim ci As New Globalization.CultureInfo("en-US")
            siswa.Verifikasi = item.Groups("Verifikasi").Value.Trim()
            Dim Pilihan As String = item.Groups("Pilihan").Value.Trim()
            If Pilihan = "I" Then
                siswa.Pilihan1 = SekolahID
                siswa.Nilai = Convert.ToDouble(item.Groups("Nilai").Value, ci)
            Else
                siswa.Pilihan2 = SekolahID
                siswa.Nilai2 = Convert.ToDouble(item.Groups("Nilai").Value, ci)
            End If

            siswaList.Add(siswa)
        Next

        If offset = 0 Then
            Dim offsetDom = getRegex(page, "<a[^>]*?show\((?<offset>\d*?)\)[^>]*?><span[^>]*?nav-last[^>]*?>")
            Dim lastOffset As Integer = offsetDom(0).Groups("offset").Value
            For nextOffset = 50 To lastOffset Step 50
                siswaList.AddRange(getDataSiswa(SekolahID, nextOffset))
            Next
        End If

        Return siswaList
    End Function

    Private Sub btnGetData_Click(sender As System.Object, e As System.EventArgs) Handles btnGetData.Click
        Dim SekolahID As String = NamaSekolahComboBox.SelectedValue.ToString()
        Dim siswaList = getDataSiswa(SekolahID, 0)
        SiswaBindingSource.DataSource = siswaList
        If Not isGetAll Then MessageBox.Show("Berhasil fetch data.")
    End Sub

    Sub insertSekolah(ByRef db As PPDCrawlerContext)
        If db Is Nothing Then
            db = New PPDCrawlerContext()
        End If

        Dim page As String = getPage("http://ppd.disdik-kotasmg.org/v12/smp/", "cassila=3e1dedcba81c8d75fe94e83fcc11890e&url_=http://ppd.disdik-kotasmg.org/v12/smp/menu/rekap/")
        Dim SekolahDOM As MatchCollection = getRegex(page, "<option[^>]*value=['""](?<id>.*?)['""][^>]*>(?<sekolah>.*?)</option>")
        For Each item As Match In SekolahDOM
            Dim sekolahID = item.Groups("id").Value
            Dim namaSekolah = item.Groups("sekolah").Value
            If sekolahID IsNot Nothing AndAlso Not String.IsNullOrEmpty(sekolahID) Then
                Dim sekolah As New Sekolah()
                sekolah.SekolahID = sekolahID
                sekolah.NamaSekolah = namaSekolah
                db.Sekolah.Add(sekolah)
            End If
        Next
        db.SaveChanges()
    End Sub

    Private Sub Form1_Load(sender As System.Object, e As System.EventArgs) Handles MyBase.Load
        Using db As New PPDCrawlerContext()
            db.Sekolah.Load()
            If db.Sekolah.Local.Count = 0 Then
                insertSekolah(db)
                db.Sekolah.Load()
            End If

            NamaSekolahComboBox.DataSource = db.Sekolah.Local.ToList()
        End Using
    End Sub

    Private Sub btnSimpanData_Click(sender As System.Object, e As System.EventArgs) Handles btnSimpanData.Click
        Using db As New PPDCrawlerContext()
            For index = 0 To GridView1.RowCount - 1
                Dim kpun = GridView1.GetRowCellValue(index, colKPUN)
                Dim pilihan1 = getDefaultIfNull(Of String)(GridView1.GetRowCellValue(index, colPilihan1))
                Dim pilihan2 = getDefaultIfNull(Of String)(GridView1.GetRowCellValue(index, colPilihan2))

                Dim siswa = If(db.Siswa.Find(kpun), db.Siswa.Add(
                               New Siswa() With {
                                   .KPUN = kpun
                               }))

                siswa.NamaSiswa = GridView1.GetRowCellValue(index, colNamaSiswa)
                siswa.SekolahAsal = GridView1.GetRowCellValue(index, colSekolahAsal)
                siswa.Verifikasi = GridView1.GetRowCellValue(index, colVerifikasi)

                If pilihan1 <> "" Then
                    siswa.Pilihan1 = pilihan1
                    siswa.Nilai = getDefaultIfNull(Of Decimal)(GridView1.GetRowCellValue(index, colNilai))
                End If
                If pilihan2 <> "" Then
                    siswa.Pilihan2 = pilihan2
                    siswa.Nilai2 = getDefaultIfNull(Of Decimal)(GridView1.GetRowCellValue(index, colNilai2))
                End If
            Next
            db.SaveChanges()
            If Not isGetAll Then MessageBox.Show("Berhasil menyimpan data.")
        End Using
    End Sub

    Private Sub btnAutomate_Click(sender As System.Object, e As System.EventArgs) Handles btnAutomate.Click
        isGetAll = True
        For index = 0 To NamaSekolahComboBox.Items.Count - 1
            NamaSekolahComboBox.SelectedIndex = index
            btnGetData_Click(sender, e)
            btnSimpanData_Click(sender, e)
        Next
        isGetAll = False
    End Sub

    Function getDefaultIfNull(Of T)(data As Object) As T
        Dim returnValue As T
        Return If(data IsNot Nothing, DirectCast(data, T), returnValue)
    End Function

    Private Sub btnOlahData_Click(sender As System.Object, e As System.EventArgs) Handles btnOlahData.Click
        Using db As New PPDCrawlerContext()
            db.Siswa.Load()
            db.Sekolah.Load()

            ' reset hasil
            For Each item In db.Siswa.Local
                item.Hasil = Nothing
            Next
            db.SaveChanges()

            ' Olah Pilihan Pertama
            For Each item In db.Sekolah.Local
                Dim kuotaPPD = item.DayaTampungPPD
                Dim sekolahID = item.SekolahID

                Dim siswa = (From s In db.Siswa.Local
                            Where s.Pilihan1 = sekolahID
                            Order By s.Nilai Descending
                            Select s).Take(kuotaPPD)

                For Each s In siswa
                    s.Hasil = sekolahID
                Next
            Next
            db.SaveChanges()

            ' Olah Pilihan Kedua
            For Each item In db.Sekolah.Local
                Dim kuotaPPD = item.DayaTampungPPD
                Dim sekolahID = item.SekolahID
                'Dim nilaiMin = (From s In db.Siswa.Local Where s.Hasil = sekolahID Order By s.Nilai Ascending Select s.Nilai).FirstOrDefault()

                'Dim siswa = (From s In db.Siswa.Local
                '             Where s.Hasil Is Nothing AndAlso s.Pilihan2 = sekolahID AndAlso s.Nilai >= nilaiMin
                '             Select s)
                Dim siswa = From s In db.Siswa.Local
                            Where s.Hasil = Nothing AndAlso s.Pilihan2 = sekolahID
                            Select s

                For Each s In siswa
                    s.Hasil = sekolahID
                Next
            Next
            db.SaveChanges()

            ' Olah yg akan Gagal di Pilihan Pertama setelah Pilihan Kedua diproses
            For Each Item In db.Sekolah.Local
                Dim kuotaPPD = Item.DayaTampungPPD
                Dim sekolahID = Item.SekolahID

                Dim siswaLolos = (From s In db.Siswa.Local
                                 Where s.Hasil = sekolahID AndAlso s.Pilihan1 = s.Hasil
                                 Order By s.Nilai Descending
                                 Select s.KPUN).Take(kuotaPPD).ToArray()

                Dim siswa = From s In db.Siswa.Local
                            Where Not siswaLolos.Contains(s.KPUN) AndAlso s.Hasil = sekolahID
                            Select s

                For Each s In siswa
                    s.Hasil = s.Pilihan2
                Next
                db.SaveChanges()
            Next

            ' Hapus Hasil yg tidak perlu
            'For Each item In db.Sekolah.Local
            '    Dim kuotaPPD = item.DayaTampungPPD
            '    Dim sekolahID = item.SekolahID
            '    Dim nilaiMin = (From s In db.Siswa.Local Where s.Hasil = sekolahID Order By s.Nilai Descending Select s).Take(kuotaPPD)
            '    Dim min = (From s In nilaiMin Order By s.Nilai Ascending Select s.Nilai).FirstOrDefault()

            '    Dim siswa = (From s In db.Siswa.Local Where s.Nilai < min AndAlso s.Hasil = sekolahID Select s)
            '    For Each s In siswa
            '        s.Hasil = Nothing
            '    Next
            'Next
            'db.SaveChanges()

            MessageBox.Show("Berhasil menyimpan data.")
        End Using
    End Sub

    Private Sub btnHasilOlahData_Click(sender As System.Object, e As System.EventArgs) Handles btnHasilOlahData.Click
        Using db As New PPDCrawlerContext()
            Dim sekolahID = cmbSekolahHasil.SelectedValue.ToString()

            Dim siswa = From s In db.Siswa Where s.Hasil = sekolahID Order By s.Nilai Descending Select s

            SiswaBindingSource.DataSource = siswa.ToList()
        End Using
    End Sub

    Private Sub txtNilai_KeyDown(sender As System.Object, e As System.Windows.Forms.KeyEventArgs) Handles txtNilai.KeyDown
        If e.KeyCode = Keys.Enter Then
            Using db As New PPDCrawlerContext()
                Dim nilai = Convert.ToDecimal(txtNilai.Text)
                Dim sekolahID = From s In db.Siswa Where s.Hasil IsNot Nothing AndAlso s.Nilai <= nilai Select s.Hasil Distinct

                Dim sekolah = From s In db.Sekolah Where sekolahID.Contains(s.SekolahID) Select s

                cmbSekolahHasil.DataSource = sekolah.ToList()
            End Using
        End If

    End Sub

    Private Sub btnHasilOlahDataKuota_Click(sender As System.Object, e As System.EventArgs) Handles btnHasilOlahDataKuota.Click
        Using db As New PPDCrawlerContext()
            Dim sekolahID = cmbSekolahHasil.SelectedValue.ToString()

            Dim kuotaPPD = (From s In db.Sekolah
                           Where s.SekolahID = sekolahID
                           Select s.DayaTampungPPD).FirstOrDefault()

            Dim siswa = (From s In db.Siswa Where s.Hasil = sekolahID Order By s.Nilai Descending Select s).Take(kuotaPPD)

            SiswaBindingSource.DataSource = siswa.ToList()
        End Using
    End Sub
End Class

Public Class Sekolah
    Public Property SekolahID As String
    Public Property NamaSekolah As String
    Public Property DayaTampungPPD As Integer
End Class

Public Class Siswa
    <Key()>
    Public Property KPUN As String
    Public Property NamaSiswa As String
    Public Property SekolahAsal As String
    Public Property Verifikasi As String
    Public Property Nilai As Decimal
    Public Property Pilihan1 As String
    Public Property Nilai2 As Decimal
    Public Property Pilihan2 As String
    Public Property Hasil As String
End Class

Public Class PPDCrawlerContext
    Inherits DbContext

    Public Sub New()
        MyBase.New("name=PPDCrawlerContext")
    End Sub

    Public Property Sekolah As DbSet(Of Sekolah)
    Public Property Siswa As DbSet(Of Siswa)
End Class
